From c64fd71ede512f648442806db571df6567b514df Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 26 Aug 2014 10:36:11 -0700 Subject: [PATCH] Revert "Revert "Use libgit2 for driving git instead of the CLI"" This reverts commit 04440bb035082d4163f2211af447b8d67e077ccb. Conflicts: src/cargo/sources/git/utils.rs --- Cargo.lock | 23 +++ Cargo.toml | 5 +- configure | 2 + src/cargo/core/resolver.rs | 4 +- src/cargo/core/source.rs | 4 +- src/cargo/lib.rs | 5 +- src/cargo/ops/cargo_new.rs | 32 ++-- src/cargo/sources/git/source.rs | 2 +- src/cargo/sources/git/utils.rs | 270 +++++++++++++++++--------------- src/cargo/util/errors.rs | 7 + src/cargo/util/to_url.rs | 8 + 11 files changed, 211 insertions(+), 151 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f5ea5f24..6cb208e40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,6 +5,7 @@ dependencies = [ "docopt 0.6.0 (git+https://github.com/burntsushi/docopt.rs#fc7ba2f1a5a351f7874257d880223d2ff5c75d36)", "docopt_macros 0.6.0 (git+https://github.com/burntsushi/docopt.rs#fc7ba2f1a5a351f7874257d880223d2ff5c75d36)", "flate2 0.0.1 (git+https://github.com/alexcrichton/flate2-rs#12593d1b9ccf09c2eabac176a6e233b171eed843)", + "git2 0.0.1 (git+https://github.com/alexcrichton/git2-rs#66ddeb423f5534817ec254b54eaeb50e506f87ec)", "hamcrest 0.1.0 (git+https://github.com/carllerche/hamcrest-rust.git#f0fd1546b0a7a278a12658ab8602b5c827cc3a42)", "semver 0.0.1 (git+https://github.com/rust-lang/semver#c78b40d7fdf8acd99b503e6ce394fbcf9eb8982f)", "tar 0.0.1 (git+https://github.com/alexcrichton/tar-rs#689bbc003ae47feae5bc99c53b56736e4ad994ba)", @@ -31,15 +32,37 @@ version = "0.1.0" source = "git+https://github.com/lifthrasiir/rust-encoding#7e7950ddbd46428a439db3e2594fa78f0972ef2e" [[package]] +<<<<<<< HEAD name = "flate2" version = "0.0.1" source = "git+https://github.com/alexcrichton/flate2-rs#12593d1b9ccf09c2eabac176a6e233b171eed843" +======= +name = "git2" +version = "0.0.1" +source = "git+https://github.com/alexcrichton/git2-rs#66ddeb423f5534817ec254b54eaeb50e506f87ec" +dependencies = [ + "libgit2 0.0.1 (git+https://github.com/alexcrichton/git2-rs#66ddeb423f5534817ec254b54eaeb50e506f87ec)", +] +>>>>>>> Revert "Revert "Use libgit2 for driving git instead of the CLI"" [[package]] name = "hamcrest" version = "0.1.0" source = "git+https://github.com/carllerche/hamcrest-rust.git#f0fd1546b0a7a278a12658ab8602b5c827cc3a42" +[[package]] +name = "libgit2" +version = "0.0.1" +source = "git+https://github.com/alexcrichton/git2-rs#66ddeb423f5534817ec254b54eaeb50e506f87ec" +dependencies = [ + "link-config 0.0.1 (git+https://github.com/alexcrichton/link-config#f08103ea7d2e2d3369c2c5e66b0220c8d16b92c9)", +] + +[[package]] +name = "link-config" +version = "0.0.1" +source = "git+https://github.com/alexcrichton/link-config#f08103ea7d2e2d3369c2c5e66b0220c8d16b92c9" + [[package]] name = "semver" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index e393cb5c9..f3ac95e0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ git = "https://github.com/burntsushi/docopt.rs" [dependencies.toml] git = "https://github.com/alexcrichton/toml-rs" -[dependencies.hamcrest] +[dev-dependencies.hamcrest] git = "https://github.com/carllerche/hamcrest-rust.git" [dependencies.url] @@ -32,6 +32,9 @@ git = "https://github.com/alexcrichton/tar-rs" [dependencies.flate2] git = "https://github.com/alexcrichton/flate2-rs" +[dependencies.git2] +git = "https://github.com/alexcrichton/git2-rs" + [[bin]] name = "cargo" test = false diff --git a/configure b/configure index f9a34c04e..a263e55c4 100755 --- a/configure +++ b/configure @@ -262,6 +262,8 @@ need_cmd date need_cmd tr need_cmd sed need_cmd file +need_cmd cmake +need_cmd pkg-config CFG_SRC_DIR="$(cd $(dirname $0) && pwd)/" CFG_BUILD_DIR="$(pwd)/" diff --git a/src/cargo/core/resolver.rs b/src/cargo/core/resolver.rs index 43cb2e3b4..92d81d8dd 100644 --- a/src/cargo/core/resolver.rs +++ b/src/cargo/core/resolver.rs @@ -2,7 +2,8 @@ use std::collections::HashMap; use std::fmt; use serialize::{Encodable, Encoder, Decodable, Decoder}; -use util::graph::{Nodes,Edges}; +use util::profile; +use util::graph::{Nodes, Edges}; use core::{ Dependency, @@ -238,6 +239,7 @@ impl<'a, R: Registry> Context<'a, R> { pub fn resolve(root: &PackageId, deps: &[Dependency], registry: &mut R) -> CargoResult { log!(5, "resolve; deps={}", deps); + let _p = profile::start(format!("resolving: {}", root)); let mut context = Context::new(registry, root.clone()); try!(resolve_deps(root, deps, &mut context)); diff --git a/src/cargo/core/source.rs b/src/cargo/core/source.rs index 69767edc4..7c1b2f11f 100644 --- a/src/cargo/core/source.rs +++ b/src/cargo/core/source.rs @@ -211,9 +211,7 @@ impl SourceId { // Pass absolute path pub fn for_path(path: &Path) -> CargoResult { - let url = try!(Url::from_file_path(path).map_err(|()| { - human(format!("not a valid path for a URL: {}", path.display())) - })); + let url = try!(path.to_url().map_err(human)); Ok(SourceId::new(PathKind, url)) } diff --git a/src/cargo/lib.rs b/src/cargo/lib.rs index ff95e2d87..8144eb396 100644 --- a/src/cargo/lib.rs +++ b/src/cargo/lib.rs @@ -13,12 +13,13 @@ extern crate time; #[phase(plugin)] extern crate regex_macros; #[phase(plugin, link)] extern crate log; -extern crate semver; extern crate docopt; extern crate flate2; +extern crate git2; +extern crate semver; extern crate tar; -extern crate url; extern crate toml; +extern crate url; #[cfg(test)] extern crate hamcrest; use std::os; diff --git a/src/cargo/ops/cargo_new.rs b/src/cargo/ops/cargo_new.rs index 53b462ae2..9ff86e79b 100644 --- a/src/cargo/ops/cargo_new.rs +++ b/src/cargo/ops/cargo_new.rs @@ -1,12 +1,10 @@ use std::os; use std::io::{mod, fs, File}; -use util::{CargoResult, human, ChainError, process}; -use core::shell::MultiShell; +use git2::{Repository, Config}; -macro_rules! git( ($($a:expr),*) => ({ - process("git") $(.arg($a))* .exec_with_output() -}) ) +use util::{CargoResult, human, ChainError}; +use core::shell::MultiShell; pub struct NewOptions<'a> { pub git: bool, @@ -30,7 +28,7 @@ pub fn new(opts: NewOptions, _shell: &mut MultiShell) -> CargoResult<()> { fn mk(path: &Path, name: &str, opts: &NewOptions) -> CargoResult<()> { if opts.git { - try!(git!("init", path)); + try!(Repository::init(path)); let mut gitignore = "/target\n".to_string(); if !opts.bin { gitignore.push_str("/Cargo.lock\n"); @@ -69,19 +67,17 @@ fn it_works() { } fn discover_author() -> CargoResult { - let name = match git!("config", "user.name") { - Ok(out) => String::from_utf8_lossy(out.output.as_slice()).into_string(), - Err(..) => match os::getenv("USER") { - Some(user) => user, - None => return Err(human("could not determine the current user, \ - please set $USER")) - } - }; - - let email = match git!("config", "user.email") { - Ok(out) => Some(String::from_utf8_lossy(out.output.as_slice()).into_string()), - Err(..) => None, + let git_config = Config::open_default().ok(); + let git_config = git_config.as_ref(); + let name = git_config.and_then(|g| g.get_str("user.name").ok()) + .map(|s| s.to_string()) + .or_else(|| os::getenv("USER")); + let name = match name { + Some(name) => name, + None => return Err(human("could not determine the current user, \ + please set $USER")) }; + let email = git_config.and_then(|g| g.get_str("user.email").ok()); let name = name.as_slice().trim().to_string(); let email = email.map(|s| s.as_slice().trim().to_string()); diff --git a/src/cargo/sources/git/source.rs b/src/cargo/sources/git/source.rs index 906f5140e..ff8ef9ad9 100644 --- a/src/cargo/sources/git/source.rs +++ b/src/cargo/sources/git/source.rs @@ -171,7 +171,7 @@ impl<'a, 'b> Source for GitSource<'a, 'b> { let rev = try!(repo.rev_for(self.reference.as_slice())); (repo, rev) } else { - (self.remote.db_at(&self.db_path), actual_rev.unwrap()) + (try!(self.remote.db_at(&self.db_path)), actual_rev.unwrap()) }; try!(repo.copy_to(actual_rev.clone(), &self.checkout_path)); diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index b673d631d..1158d0c8b 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -3,8 +3,9 @@ use std::io::{UserDir}; use std::io::fs::{mkdir_recursive,rmdir_recursive}; use serialize::{Encodable,Encoder}; use url::Url; +use git2; -use util::{CargoResult, ChainError, ProcessBuilder, process, human}; +use util::{CargoResult, ChainError, human, ToUrl, internal, Require}; #[deriving(PartialEq,Clone,Encodable)] pub enum GitReference { @@ -53,22 +54,6 @@ impl Show for GitRevision { } } -macro_rules! git( - ($config:expr, $($arg:expr),+) => ( - try!(git_inherit(&$config, process("git")$(.arg($arg))*)) - ) -) - -macro_rules! git_output( - ($config:expr, $($arg:expr),*) => ({ - try!(git_output(&$config, process("git")$(.arg($arg))*)) - }) -) - -macro_rules! errln( - ($($arg:tt)*) => (let _ = writeln!(::std::io::stdio::stderr(), $($arg)*)) -) - /// GitRemote represents a remote repository. It gets cloned into a local /// GitDatabase. #[deriving(PartialEq,Clone,Show)] @@ -91,10 +76,10 @@ impl> Encodable for GitRemote { /// GitDatabase is a local clone of a remote repository's database. Multiple /// GitCheckouts can be cloned from this GitDatabase. -#[deriving(PartialEq,Clone)] pub struct GitDatabase { remote: GitRemote, path: Path, + repo: git2::Repository, } #[deriving(Encodable)] @@ -115,25 +100,29 @@ impl> Encodable for GitDatabase { /// GitCheckout is a local checkout of a particular revision. Calling /// `clone_into` with a reference will resolve the reference into a revision, /// and return a CargoError if no revision for that reference was found. -pub struct GitCheckout { - database: GitDatabase, +pub struct GitCheckout<'a> { + database: &'a GitDatabase, location: Path, revision: GitRevision, + repo: git2::Repository, } #[deriving(Encodable)] pub struct EncodableGitCheckout { - database: GitDatabase, + database: EncodableGitDatabase, location: String, revision: String, } -impl> Encodable for GitCheckout { +impl<'a, E, S: Encoder> Encodable for GitCheckout<'a> { fn encode(&self, s: &mut S) -> Result<(), E> { EncodableGitCheckout { - database: self.database.clone(), location: self.location.display().to_string(), - revision: self.revision.to_string() + revision: self.revision.to_string(), + database: EncodableGitDatabase { + remote: self.database.remote.clone(), + path: self.database.path.display().to_string(), + }, }.encode(s) } } @@ -151,50 +140,53 @@ impl GitRemote { pub fn rev_for(&self, path: &Path, reference: S) -> CargoResult { - // We simultaneously want to transform the reference into a resolved - // revision as well as verify that the reference itself is inside the - // repository. Sadly for a 40-character SHA1 the call to `rev-parse` - // will *always* return the same string with a 0 exit status, regardless - // of whether it's present in the database. - // - // Later versions of git introduced a syntax for this query via - // `$sha1^{object}`, but older versions of git do not support this. To - // get around this limitation, we chop 40-character sha revisions to 39 - // characters to get an error'd exit status if the revision is indeed - // not present. - let mut reference = reference.as_slice(); - if reference.len() == 40 { - reference = reference.slice_to(39); - } - Ok(GitRevision(git_output!(*path, "rev-parse", reference))) + let db = try!(self.db_at(path)); + db.rev_for(reference) } pub fn checkout(&self, into: &Path) -> CargoResult { - if into.exists() { - try!(self.fetch_into(into)); + let repo = if into.exists() { + let r = try!(git2::Repository::open(into)); + try!(self.fetch_into(&r).chain_error(|| { + internal(format!("failed to fetch into {}", into.display())) + })); + r } else { - try!(self.clone_into(into)); - } + try!(self.clone_into(into).chain_error(|| { + internal(format!("failed to clone into: {}", into.display())) + })) + }; - Ok(GitDatabase { remote: self.clone(), path: into.clone() }) + Ok(GitDatabase { remote: self.clone(), path: into.clone(), repo: repo }) } - pub fn db_at(&self, db_path: &Path) -> GitDatabase { - GitDatabase { remote: self.clone(), path: db_path.clone() } + pub fn db_at(&self, db_path: &Path) -> CargoResult { + let repo = try!(git2::Repository::open(db_path)); + Ok(GitDatabase { + remote: self.clone(), + path: db_path.clone(), + repo: repo, + }) } - fn fetch_into(&self, path: &Path) -> CargoResult<()> { - Ok(git!(*path, "fetch", "--force", "--quiet", "--tags", - self.url.to_string(), "refs/heads/*:refs/heads/*")) + fn fetch_into(&self, dst: &git2::Repository) -> CargoResult<()> { + let url = self.url.to_string(); + let refspec = "refs/heads/*:refs/heads/*"; + let mut remote = try!(dst.remote_create_anonymous(url.as_slice(), + refspec)); + try!(remote.add_fetch("refs/tags/*:refs/tags/*")); + let sig = try!(git2::Signature::default(dst)); + try!(remote.fetch(&sig, None)); + Ok(()) } - fn clone_into(&self, path: &Path) -> CargoResult<()> { - let dirname = Path::new(path.dirname()); - - try!(mkdir_recursive(path, UserDir)); - - Ok(git!(dirname, "clone", self.url.to_string(), path, "--bare", - "--no-hardlinks", "--quiet")) + fn clone_into(&self, dst: &Path) -> CargoResult { + let url = self.url.to_string(); + try!(mkdir_recursive(dst, UserDir)); + let repo = try!(git2::build::RepoBuilder::new().bare(true) + .hardlinks(false) + .clone(url.as_slice(), dst)); + Ok(repo) } } @@ -205,119 +197,147 @@ impl GitDatabase { pub fn copy_to(&self, rev: GitRevision, dest: &Path) -> CargoResult { - - if dest.exists() { - match self.remote.rev_for(dest, "HEAD") { - Ok(ref head) if rev == *head => { - return Ok(GitCheckout::new(dest, self.clone(), rev.clone())); + match git2::Repository::open(dest) { + Ok(repo) => { + let is_fresh = match repo.revparse_single("HEAD") { + Ok(head) => head.id().to_string() == rev.to_string(), + _ => false, + }; + if is_fresh { + return Ok(GitCheckout::new(dest, self, rev, repo)) } - _ => {} } + _ => {} } - GitCheckout::clone_into(dest, self.clone(), rev.clone()) + GitCheckout::clone_into(dest, self, rev) } pub fn rev_for(&self, reference: S) -> CargoResult { - self.remote.rev_for(&self.path, reference) + let rev = try!(self.repo.revparse_single(reference.as_slice())); + Ok(GitRevision(rev.id().to_string())) } pub fn has_ref(&self, reference: S) -> CargoResult<()> { - git_output!(self.path, "rev-parse", "--verify", reference.as_slice()); + try!(self.repo.revparse_single(reference.as_slice())); Ok(()) } } -impl GitCheckout { - fn new(path: &Path, database: GitDatabase, revision: GitRevision) - -> GitCheckout +impl<'a> GitCheckout<'a> { + fn new<'a>(path: &Path, database: &'a GitDatabase, revision: GitRevision, + repo: git2::Repository) + -> GitCheckout<'a> { GitCheckout { location: path.clone(), database: database, revision: revision, + repo: repo, } } - fn clone_into(into: &Path, database: GitDatabase, - revision: GitRevision) - -> CargoResult + fn clone_into<'a>(into: &Path, database: &'a GitDatabase, + revision: GitRevision) + -> CargoResult> { - let checkout = GitCheckout::new(into, database, revision); + let repo = try!(GitCheckout::clone_repo(database.get_path(), into)); + let checkout = GitCheckout::new(into, database, revision, repo); - try!(checkout.clone_repo()); + try!(checkout.reset()); try!(checkout.update_submodules()); Ok(checkout) } - fn get_source(&self) -> &Path { - self.database.get_path() - } - pub fn get_rev(&self) -> &str { self.revision.as_slice() } - fn clone_repo(&self) -> CargoResult<()> { - let dirname = Path::new(self.location.dirname()); + fn clone_repo(source: &Path, into: &Path) -> CargoResult { + let dirname = into.dir_path(); try!(mkdir_recursive(&dirname, UserDir).chain_error(|| { - human(format!("Couldn't mkdir {}", - Path::new(self.location.dirname()).display())) + human(format!("Couldn't mkdir {}", dirname.display())) })); - if self.location.exists() { - try!(rmdir_recursive(&self.location).chain_error(|| { - human(format!("Couldn't rmdir {}", - Path::new(&self.location).display())) + if into.exists() { + try!(rmdir_recursive(into).chain_error(|| { + human(format!("Couldn't rmdir {}", into.display())) })); } - git!(dirname, "clone", "--no-checkout", "--quiet", - self.get_source(), &self.location); - try!(self.reset()); - - Ok(()) + let url = try!(source.to_url().map_err(human)); + let url = url.to_string(); + let repo = try!(git2::Repository::clone(url.as_slice(), + into).chain_error(|| { + internal(format!("failed to clone {} into {}", source.display(), + into.display())) + })); + Ok(repo) } fn reset(&self) -> CargoResult<()> { - Ok(git!(self.location, "reset", "-q", "--hard", - self.revision.as_slice())) + info!("reset {} to {}", self.repo.path().display(), + self.revision.as_slice()); + let sig = try!(git2::Signature::default(&self.repo)); + let oid = try!(git2::Oid::from_str(self.revision.as_slice())); + let object = try!(git2::Object::lookup(&self.repo, oid, None)); + try!(self.repo.reset(&object, git2::Hard, &sig, None)); + Ok(()) } fn update_submodules(&self) -> CargoResult<()> { - git!(self.location, "submodule", "sync", "--quiet"); - // Sadly older versions of git don't actually respect --quiet for *all* - // operations and still print some thing here and there. - git_output!(self.location, "submodule", "update", "--init", - "--recursive", "--quiet"); - Ok(()) + let sig = try!(git2::Signature::default(&self.repo)); + return update_submodules(&self.repo, &sig); + + fn update_submodules(repo: &git2::Repository, + sig: &git2::Signature) -> CargoResult<()> { + info!("update submodules for: {}", repo.path().display()); + + for mut child in try!(repo.submodules()).move_iter() { + try!(child.init(false)); + + // A submodule which is listed in .gitmodules but not actually + // checked out will not have a head id, so we should ignore it. + let head = match child.head_id() { + Some(head) => head, + None => continue, + }; + + // If the submodule hasn't been checked out yet, we need to + // clone it. If it has been checked out and the head is the same + // as the submodule's head, then we can bail out and go to the + // next submodule. + let repo = match child.open() { + Ok(repo) => { + if child.head_id() == try!(repo.head()).target() { + continue + } + repo + } + Err(..) => { + let path = repo.path().dir_path().join(child.path()); + let url = try!(child.url().require(|| { + internal("invalid submodule url") + })); + try!(git2::Repository::clone(url, &path)) + } + }; + + // Fetch data from origin and reset to the head commit + let url = try!(child.url().require(|| { + internal("repo with non-utf8 url") + })); + let refspec = "refs/heads/*:refs/heads/*"; + let mut remote = try!(repo.remote_create_anonymous(url, refspec)); + try!(remote.fetch(sig, None)); + + let obj = try!(git2::Object::lookup(&repo, head, None)); + try!(repo.reset(&obj, git2::Hard, sig, None)); + try!(update_submodules(&repo, sig)); + } + Ok(()) + } } } - -fn git(path: &Path, cmd: ProcessBuilder) -> ProcessBuilder { - debug!("Executing {} @ {}", cmd, path.display()); - - cmd.cwd(path.clone()) -} - -fn git_inherit(path: &Path, cmd: ProcessBuilder) -> CargoResult<()> { - let cmd = git(path, cmd); - cmd.exec().chain_error(|| { - human(format!("Executing {} failed", cmd)) - }) -} - -fn git_output(path: &Path, cmd: ProcessBuilder) -> CargoResult { - let cmd = git(path, cmd); - let output = try!(cmd.exec_with_output().chain_error(|| - human(format!("Executing {} failed", cmd)))); - - Ok(to_str(output.output.as_slice()).as_slice().trim_right().to_string()) -} - -fn to_str(vec: &[u8]) -> String { - String::from_utf8_lossy(vec).into_string() -} - diff --git a/src/cargo/util/errors.rs b/src/cargo/util/errors.rs index 5ec2f711c..a33ca305c 100644 --- a/src/cargo/util/errors.rs +++ b/src/cargo/util/errors.rs @@ -6,6 +6,7 @@ use std::str; use docopt; use toml::Error as TomlError; use url; +use git2; pub trait CargoError: Send { fn description(&self) -> String; @@ -293,6 +294,12 @@ impl CargoError for url::ParseError { from_error!(url::ParseError) +impl CargoError for git2::Error { + fn description(&self) -> String { self.to_string() } +} + +from_error!(git2::Error) + impl CliError { pub fn new(error: S, code: uint) -> CliError { let error = human(error.as_slice().to_string()); diff --git a/src/cargo/util/to_url.rs b/src/cargo/util/to_url.rs index c99ba4d27..f4fe44442 100644 --- a/src/cargo/util/to_url.rs +++ b/src/cargo/util/to_url.rs @@ -24,6 +24,14 @@ impl<'a> ToUrl for &'a str { } } +impl<'a> ToUrl for &'a Path { + fn to_url(self) -> Result { + Url::from_file_path(self).map_err(|()| { + format!("invalid path url `{}`", self.display()) + }) + } +} + fn mapper(s: &str) -> url::SchemeType { match s { "git" => url::RelativeScheme(9418), -- 2.30.2